BEGIN{
    # Keep track of current experiment (as results span multiple lines)
    lastexp="NONE"

    # Keep track of mismatches
    mismatches=""

    # Keep track of data
    smallestfile=99999999
    largestfile=0
    largestdsfiles=0
    currentdsfiles=0

    # Array indexes
    labelidx=0
    expcountidx=1
    filecountidx=2
    okidx=3
    notokidx=4
    rerunnotokidx=5
    fsdataidx=6
    ddtimeidx=7
    fstimeidx=8
    dbtimeidx=9

    # Set up array test labels
    tests=10
    testarray[0,labelidx]="Totals"
    testarray[1,labelidx]="test_1"
    testarray[2,labelidx]="test_2"
    testarray[3,labelidx]="test_3"
    testarray[4,labelidx]="test_4"
    testarray[5,labelidx]="test_5"
    testarray[6,labelidx]="test_6"
    testarray[7,labelidx]="test_100000"
    testarray[8,labelidx]="test_400000"
    testarray[9,labelidx]="copyonly_5"
    testarray[10,labelidx]="copyonly_100000"

    # Set up test counters in array
    for (t=0;t<=tests;t++)
    {
      testarray[t,expcountidx]=0
      testarray[t,filecountidx]=0
      testarray[t,okidx]=0
      testarray[t,notokidx]=0
      testarray[t,rerunnotokidx]=0
      testarray[t,fsdataidx]=0
      testarray[t,ddtimeidx]=0
      testarray[t,fstimeidx]=0
      testarray[t,dbtimeidx]=0
    }
}

{
    for (t = 1; t <= tests; t++) {
      if ($14 == testarray[t,labelidx]) {
        # Add one to test counters
        # This test
        testarray[t,filecountidx]++
        testarray[t,fsdataidx]+=$6
        testarray[t,ddtimeidx]+=$8
        testarray[t,fstimeidx]+=$9
        testarray[t,dbtimeidx]+=$10
        # Total
        testarray[0,filecountidx]++
        testarray[0,fsdataidx]+=$6
        testarray[0,ddtimeidx]+=$8
        testarray[0,fstimeidx]+=$9
        testarray[0,dbtimeidx]+=$10

        if ($6<smallestfile) smallestfile=$6
        if ($6>largestfile) largestfile=$6

        # Only change these counters if it is a different experiment
        if ($4!=lastexp){
          testarray[t,expcountidx]++  # This test
          testarray[0,expcountidx]++  # Total

          # Reset data set file counter
          currentdsfiles=0

          # We have finished this result. Record the experiment we were working on.
          lastexp=$4
        }

        # Record data set size if it is bigger
        if (++currentdsfiles > largestdsfiles) largestdsfiles = currentdsfiles

        # Check if the results matched
        if ($11=="Mismatch" || $12=="Mismatch" || $13=="Mismatch") {
          testarray[t,notokidx]++  # This test
          testarray[0,notokidx]++  # Total

          # Rerun tests
          reruncmd = " bash \"$SCRIPTSPATH/rerunmatch.sh\" " $0
          reruncmd | getline rerun
          close(reruncmd)
          split(rerun,rerunarr,FS)

          zerofile=( ($1!="da39a3ee5e6b4b0d3255bfef95601890afd80709" && $3=="da39a3ee5e6b4b0d3255bfef95601890afd80709" && $6!=0 && $7==0) ? "?" : " ")
          if (rerunarr[11]=="Match" && rerunarr[12]=="Match" && rerunarr[13]=="Match"){
            rerunok="+"
          }
          else{
            testarray[t,rerunnotokidx]++
            testarray[0,rerunnotokidx]++
            rerunok=" "
          }
          mismatches=mismatches "\n*" zerofile rerunok " " $0
        }
        else {
          testarray[t,okidx]++  # This test
          testarray[0,okidx]++  # Total
        }
      }
    }
}

END{
  printf "\n========================="
  printf "\nHDC TEST RESULTS ANALYSIS"
  printf "\n=========================\n"
  printf "\n-------------------------------------------------------------------------------------------------------------\n"
  printffmt="%-16s %-5s %-14s %-9s %-13s %-27s %-9s %-10s\n"
  printf printffmt, "Test label","Runs", "Files created", "Files OK", "Files Not OK", "Files not OK after recheck",     "Avg Read", "Avg Write"
  printf printffmt, "---------------","----", "-------------", "--------", "------------", "--------------------------","--------", "---------"
  for(t = 1; t <= tests; t++)
  {
    printf printffmt, testarray[t,labelidx], testarray[t,expcountidx], testarray[t,filecountidx], testarray[t,okidx], testarray[t,notokidx] (testarray[t,notokidx] > 0 ? " *" : ""), testarray[t,rerunnotokidx], ( testarray[t,filecountidx]>0 ? testarray[t,fstimeidx]/testarray[t,filecountidx] : 0 ), ( testarray[t,filecountidx]>0 ? testarray[t,ddtimeidx]/testarray[t,filecountidx] : 0 )
  }
  printf printffmt, "---------------","----", "-------------", "--------", "------------", "--------------------------","--------", "---------"
  printf printffmt, testarray[0,labelidx], testarray[0,expcountidx],  testarray[0,filecountidx], testarray[0,okidx], testarray[0,notokidx] (testarray[0,notokidx] > 0 ? " *" : ""), testarray[0,rerunnotokidx], ( testarray[0,filecountidx]>0 ? testarray[0,fstimeidx]/testarray[0,filecountidx] : 0 ), ( testarray[0,filecountidx]>0 ? testarray[0,ddtimeidx]/testarray[0,filecountidx] : 0 )
  printf "-------------------------------------------------------------------------------------------------------------"
  if (testarray[0,notokidx]>0) {
	printf "\n\nMismatches found:\n"
	printf mismatches
	printf "\n"
	printf "\n*   A mismatch between hashes or file sizes have been found"
	printf "\n ?  A DB hash mismatch where the DB match is da39a3ee5e6b4b0d3255bfef95601890afd80709 and the DB file size is 0 may mean that the SELECT in the test script was ran too early. These have been marked with '?'"
	printf "\n  + The results have been recalculated and the hash and file sizes are now correct (the data was probably still being written and the system has now caught up)"
  }
  printf "\n\n\nSummary:\n\n"
  totalmb=testarray[0,fsdataidx]/1024/1024
  printf "%-17s: %-15s\n", "Total files", testarray[0,filecountidx] " files"
  printf "%-17s: %-15s %-15s %-15s\n", "Writing time", testarray[0,ddtimeidx] " ms", "(" testarray[0,ddtimeidx]/1000 " s)", totalmb/(testarray[0,ddtimeidx]/1000) " MB/s"
  printf "%-17s: %-15s %-15s %-15s\n", "Reading time", testarray[0,fstimeidx] " ms", "(" testarray[0,fstimeidx]/1000 " s)", totalmb/(testarray[0,fstimeidx]/1000) " MB/s"
  printf "%-17s: %-15s %-15s %-15s\n", "Hashing time", testarray[0,dbtimeidx] " ms", "(" testarray[0,dbtimeidx]/1000 " s)", totalmb/(testarray[0,dbtimeidx]/1000) " MB/s"
  printf "%-17s: %-15s %-15s\n", "Largest file", largestfile " B", "(" largestfile/1024/1024" MB)"
  printf "%-17s: %-15s %-15s\n", "Smallest file", smallestfile " B", "(" smallestfile/1024/1024" MB)"
  printf "%-17s: %-15s\n", "Largest data set", largestdsfiles " files"
  printf "%-17s: %-15s %-15s\n", "Total data", testarray[0,fsdataidx] " B", "(" totalmb/1024 " GB)"
  printf "\n\nAnalysis complete. Processed %s lines. Found %s mismatches, %s after rechecking.\n", NR, testarray[0,notokidx], testarray[0,rerunnotokidx]
  printf "\n"
}
